package cn.lyjuan.first.hadoop.demo.ch04;

import cn.lyjuan.base.util.RandomUtils;
import cn.lyjuan.first.hadoop.demo.enums.ChNameEnum;
import cn.lyjuan.first.hadoop.util.FileUtil;
import cn.lyjuan.first.hadoop.util.HDFSUtil;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;

/**
 * 统计文本中单词个数：客户端、作业驱动类
 */
public class Ch04S01WordCount {
    /**
     * Map计算任务
     * 完成键值映射关系，其实就是解析数据为键值对
     */
    public static class TokenizerMapper extends Mapper<LongWritable, Text, Text, IntWritable> {

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            Text keyOut = null;
            IntWritable valueOut = new IntWritable(1);
            /**
             * 构造一个用来解析输入 value值的StringTokenizer对象
             * Java默认的分隔符是" "、"\t"、"\n(换行)"、"\r(回车)"
             */
            StringTokenizer token = new StringTokenizer(value.toString());
            while (token.hasMoreTokens()) {
                keyOut = new Text(token.nextToken());
            }
            // map方法输出键值对：输出每个被拆分出来的单词(key)，以及计数(Value) 此处只会为1
            context.write(keyOut, valueOut);
        }
    }

    /**
     * map计算结果排序、合并
     */
    public static class IntSumReduce extends Reducer<Text, IntWritable, Text, IntWritable> {
        private IntWritable result = new IntWritable();

        @Override
        protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
            int sum = 0;
            // 对一个key的多个结果进行合并、计算
            for (IntWritable val : values) {
                sum += val.get();
            }
            result.set(sum);
            // 输出结果
            context.write(key, result);
        }
    }

    private static List<Path> inFiles = new ArrayList<>();

    private static FileSystem fs = HDFSUtil.fs();

    private static String REMOTE_PREFIX = HDFSUtil.HADOOP_URI + FileUtil.remotePath(ChNameEnum.CH04, Ch04S01WordCount.class);

    /**
     * 生成输入文件
     */
    public static void loadInputFiles() throws Exception {

        String localPrefix = FileUtil.docPath(ChNameEnum.CH04, Ch04S01WordCount.class, "source.txt");
        // 删除目录
        fs.delete(new Path(REMOTE_PREFIX), true);
        // 读取源数据内容
        BufferedReader reader = new BufferedReader(new FileReader(localPrefix));
        String line = null;
        while (null != (line = reader.readLine())) {
            Path remotePath = new Path(REMOTE_PREFIX + "/" + RandomUtils.uuid());
            inFiles.add(remotePath);
            HDFSUtil.writeRemote(remotePath, line);
            System.out.println("write remote file: " + remotePath.toString());
        }
    }

    public static void main(String[] args) throws Exception {
        // 加载源文件
        loadInputFiles();

        // 结果输出目录
        Path outPath = new Path(REMOTE_PREFIX + "/result");

        Job job = new Job(HDFSUtil.conf(), "word count");
        job.setUser(HDFSUtil.USER);
        job.setJarByClass(Ch04S01WordCount.class);
        job.setMapperClass(TokenizerMapper.class);
        job.setCombinerClass(IntSumReduce.class);
        job.setReducerClass(IntSumReduce.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        for (Path i : inFiles)
            FileInputFormat.addInputPath(job, i);
        FileOutputFormat.setOutputPath(job, outPath);
        boolean isOk = job.waitForCompletion(true);
        System.out.println("result ==> " + isOk);
        if (isOk) {// 成功时生成结果文件，报错时不生成任何文件
            System.out.println("result succ ==> " + new String(HDFSUtil.readRemote(outPath.toString() + "/_SUCCESS")));
            System.out.println("result part ==> " + new String(HDFSUtil.readRemote(outPath.toString() + "/part-r-00000")));
        }
    }
}










